/*******************************************************************************
Project:		Wealth -- Smith, Zidar, and Zwick
Last modified: 	2021-09-28
Modified by:	Blake Woods
Description:	Taken from Zwick's programs_20210225.do.
*******************************************************************************/

capture program drop build_housing_factors
program define build_housing_factors

    tempfile dq prop attom

    load_analysis_data attom
    save `attom'

    * 1. Build factors for cross section.
    *query_factors_Z
    load_analysis_data dq_factors

    gen factor_1 = totalval/taxval
    gen factor_2 = totalval2/taxval
    gen factor = cond(factor_2 > factor_1 & factor_2 != ., factor_2, factor_1)
    gen agg_valH = cond(factor_2 > factor_1 & factor_2 != ., totalval2, totalval)

    /* Version 1.0: Illinois is biased by corruption
    sum factor if state == "IL"
    local factorIL = `r(mean)'
    local factorILnew = 50
    replace factor = `factorILnew' if state == "IL"
    replace agg_valH = (`factorILnew' / `factorIL' ) * agg_valH if state == "IL"
    */

    * Version 2.0: fix other property tax rates using ATTOM collapse from most recent
    * year...possible candidates include IL, PA, DE, VT. In Downloads right now.
    merge 1:1 state using `attom', keep(3) nogen
    gen etr_cl = 1/factor
    gen etr_diff = abs(etr_cl - etr_attom)
    gen factor_cl = factor
    gen factor_attom = 1/etr_attom
    replace factor = factor_attom if etr_diff > .001
    replace agg_valH = (factor_attom / factor_cl) * agg_valH if etr_diff > .001

    egen agg_val = total(agg_valH)
    egen agg_tax = total(taxval)
    gen factor_agg = agg_val/agg_tax
    keep state factor* taxval 
    rename taxval proptax_cl
    save `dq'

    load_analysis_data proptax
    rename stateabbrev state
    rename proptaxrev_sl proptax_census
    keep statename state statefips year proptax_census proptaxrev pop
    * Recode to millions to match Corelogic
    replace proptax_census = 1e-3 * proptax_census
    merge m:1 state using `dq', keep(3) nogen
    save `prop'

    * 2. Use house price indexes to create time series of factors.
    * Load state price indexes from CoreLogic.
    tempfile factors_ts
    *make_corelogic_20191022
    load_analysis_data corelogic
    drop if year == 1976
    * Move to 2012 dollars
    gen prcH = (year == 2012) * price_index_cl
    bys state: egen price_index_2012 = max(prcH)
    replace price_index_cl = price_index_cl / price_index_2012
    drop prcH price_index_2012
    merge 1:1 state year using `prop', keep(1 3) nogen

    sort state year
    foreach v of varlist pop proptax_census {
        by state: assert `v'[_n-1] != . if year == 2016
    }

    * Estimate real estate wealth in other years
    * V1: capitalize CoreLogic prop taxes to estimate RE val
    gen realH = (year == 2012) * (factor * proptax_cl)
    * V2: capitalize Census S&L prop taxes to estimate RE val
    *gen realH = (year == 2012) * (factor * proptax_census)
    * V2 tends to be greater than V1 as CoreLogic does not have 100%
    * coverage of states. However, V2 looks to generate housing values that are
    * too large relative to USFAs.
    bys state: egen real_2012 = max(realH)
    gen realestate_val = price_index_cl * real_2012
    gen popH = (year == 2012) * pop
    bys state: egen pop_2012 = max(popH)
    gen pop_scale = pop/pop_2012
    drop realH real_2012 popH
    replace realestate_val = realestate_val * pop_scale

    * Cap factors in other years

    * Generate a scale variable since Census real estate tax revenues are on too
    * broad a base (includes commercial real estate)
    gen proptax_clH = (year == 2012) * proptax_cl
    egen agg_proptax_cl = total(proptax_clH)
    bys year: egen agg_proptax_census = total(proptax_census)
    gen proptax_scaleH = (year == 2012) * agg_proptax_cl / agg_proptax_census
    egen proptax_scale = max(proptax_scaleH)
    gen proptax_census_scale = proptax_census * proptax_scale

    * Generate factors
    gen factor_housing = realestate_val / (proptax_census_scale)
    sort state year
    replace factor_housing = (factor_housing[_n-1] + factor_housing[_n+1])/2 ///
        if year == 2001 | year == 2003

    keep year-proptax_cl factor_agg proptax_census_scale factor realestate_val pop_scale factor_housing 

    bys year: egen agg_reval = total(realestate_val) 
    bys year: egen agg_proptax = total(proptax_census_scale) 
    gen agg_factor_housing = agg_reval/agg_proptax
    save `factors_ts'

    outsheet using "$outdir/housing_factors_20200127.csv", replace comma

end
